Top Packages with Large Datasets

1 Setup

library(NNbenchmark)
library(kableExtra)
library(knitr)
options(scipen = 999)

2 Dataset to test

NNdataSummary(NNbigdatasets)

##   Dataset n_rows n_inputs n_neurons n_parameters
## 1 bWoodN1  20000        6         5           41

ht(bWoodN1)

##              x1         x2        x3        x4        x5        x6          y
## 1     0.5042050 0.05765295 0.9028320 0.4322548 0.4187889 0.3419781 -15.178295
## 2     0.9301722 0.53164816 0.4434929 0.8087859 0.6993847 0.5547075  14.048025
## 3     0.3437977 0.22566605 0.6483183 0.5155759 0.9063721 0.4779758 -20.257463
## 19998 1.0000000 1.00000000 1.0000000 1.0000000 0.0000000 1.0000000  24.227091
## 19999 1.0000000 1.00000000 1.0000000 1.0000000 1.0000000 0.0000000   5.560786
## 20000 1.0000000 1.00000000 1.0000000 1.0000000 1.0000000 1.0000000  24.309251

3 trainPredict_1pkg arguments

3.1 For all TOP5 packages

if(dir.exists("D:/GSoC2020/Results/2020run05/"))
{  
  odir <- "D:/GSoC2020/Results/2020run05/"
}else if(dir.exists("~/Documents/recherche-enseignement/code/R/NNbenchmark-project/NNtempresult/"))
{  
  odir <- "~/Documents/recherche-enseignement/code/R/NNbenchmark-project/NNtempresult/"
}else
  odir <- "."

nrep <- 10
maxit2ndorder  <-    400
maxit1storderA <-   2000
maxit1storderB <-  10000
maxit1storderC <- 100000

3.2 For each TOP5 package

Extract appropriate R code for NN configuration

pkglist <- c("nlsr", "rminer", "nnet", "validann", "MachineShop")

for(pkg in pkglist)
{
  pkgname.R <- paste0("NNbenchmark-2020-", pkg, ".R")
  pkgname.Rmd <- paste0("NNbenchmark-2020-", pkg, ".Rmd")
  configname.R <- paste0("NNbenchmark-2020-", pkg, "-Config.R")
  if(!file.exists(configname.R))
  {
    if(!file.exists(pkgname.R))
      purl(pkgname.Rmd)
    
    ConfigFile <- readLines(pkgname.R)
    ConfigFile.beg <- 28
    ConfigFile.end <- which(ConfigFile == "if(FALSE)")-1
    if(length(ConfigFile.end) == 0)
      ConfigFile.end <- 48
    
    writeLines(ConfigFile[ConfigFile.beg:ConfigFile.end], configname.R)
  }
  source(configname.R)
}

4 Launch trainPredict_1data for all packages

methodlist <- list(nlsr.method, 
                   rminer.method,
                   nnet.method, 
                   validann.method[2],
                   MachineShop.method)
pkgfunmat <- rbind(c("nlsr", "nlxb"),
                    c("rminer", "fit"),
                    c("nnet", "nnet"),
                    c("validann","ann"),
                   c("MachineShop", "fit"))
colnames(pkgfunmat) <- c("pkg", "fun")  
trainvect <- paste("NNtrain", pkgfunmat[,"pkg"], sep=".")
hypervect <- paste("hyperParams", pkgfunmat[,"pkg"], sep=".")
predvect <- paste("NNpredict", pkgfunmat[,"pkg"], sep=".")
closevect <- paste("NNclose", pkgfunmat[,"pkg"], sep=".")
startvect <- rep(NA, length(pkgfunmat[,"pkg"]))
startvect[pkgfunmat[,"pkg"] == "h2o"] <- "NNstart.h2o"
preparelist <- list(nlsr.prepareZZ, 
                    rminer.prepareZZ,
                    nnet.prepareZZ, 
                    validann.prepareZZ,
                    MachineShop.prepareZZ)
names(preparelist) <- pkgfunmat[,"pkg"]

resall <- trainPredict_1data(dset=13, method=methodlist, train=trainvect, hyper=hypervect,
                     pred=predvect, summary=NNsummary, close=closevect, 
                     start=startvect, prepare=preparelist, nrep=nrep, echo=FALSE, doplot=TRUE,
                     pkgname=pkgfunmat[,"pkg"], pkgfun=pkgfunmat[,"fun"], 
                     csvfile = TRUE, rdafile = TRUE, odir = odir)

## vn: [1] "y"   "b1"  "b2"  "b3"  "b4"  "x1"  "b5"  "x2"  "b6"  "x3"  "b7"  "x4" 
## [13] "b8"  "x5"  "b9"  "x6"  "b10" "b11" "b12" "b13" "b14" "b15" "b16" "b17"
## [25] "b18" "b19" "b20" "b21" "b22" "b23" "b24" "b25" "b26" "b27" "b28" "b29"
## [37] "b30" "b31" "b32" "b33" "b34" "b35" "b36" "b37" "b38" "b39" "b40" "b41"
## no weights
## vn: [1] "y"   "b1"  "b2"  "b3"  "b4"  "x1"  "b5"  "x2"  "b6"  "x3"  "b7"  "x4" 
## [13] "b8"  "x5"  "b9"  "x6"  "b10" "b11" "b12" "b13" "b14" "b15" "b16" "b17"
## [25] "b18" "b19" "b20" "b21" "b22" "b23" "b24" "b25" "b26" "b27" "b28" "b29"
## [37] "b30" "b31" "b32" "b33" "b34" "b35" "b36" "b37" "b38" "b39" "b40" "b41"
## no weights
## vn: [1] "y"   "b1"  "b2"  "b3"  "b4"  "x1"  "b5"  "x2"  "b6"  "x3"  "b7"  "x4" 
## [13] "b8"  "x5"  "b9"  "x6"  "b10" "b11" "b12" "b13" "b14" "b15" "b16" "b17"
## [25] "b18" "b19" "b20" "b21" "b22" "b23" "b24" "b25" "b26" "b27" "b28" "b29"
## [37] "b30" "b31" "b32" "b33" "b34" "b35" "b36" "b37" "b38" "b39" "b40" "b41"
## no weights
## vn: [1] "y"   "b1"  "b2"  "b3"  "b4"  "x1"  "b5"  "x2"  "b6"  "x3"  "b7"  "x4" 
## [13] "b8"  "x5"  "b9"  "x6"  "b10" "b11" "b12" "b13" "b14" "b15" "b16" "b17"
## [25] "b18" "b19" "b20" "b21" "b22" "b23" "b24" "b25" "b26" "b27" "b28" "b29"
## [37] "b30" "b31" "b32" "b33" "b34" "b35" "b36" "b37" "b38" "b39" "b40" "b41"
## no weights
## vn: [1] "y"   "b1"  "b2"  "b3"  "b4"  "x1"  "b5"  "x2"  "b6"  "x3"  "b7"  "x4" 
## [13] "b8"  "x5"  "b9"  "x6"  "b10" "b11" "b12" "b13" "b14" "b15" "b16" "b17"
## [25] "b18" "b19" "b20" "b21" "b22" "b23" "b24" "b25" "b26" "b27" "b28" "b29"
## [37] "b30" "b31" "b32" "b33" "b34" "b35" "b36" "b37" "b38" "b39" "b40" "b41"
## no weights
## vn: [1] "y"   "b1"  "b2"  "b3"  "b4"  "x1"  "b5"  "x2"  "b6"  "x3"  "b7"  "x4" 
## [13] "b8"  "x5"  "b9"  "x6"  "b10" "b11" "b12" "b13" "b14" "b15" "b16" "b17"
## [25] "b18" "b19" "b20" "b21" "b22" "b23" "b24" "b25" "b26" "b27" "b28" "b29"
## [37] "b30" "b31" "b32" "b33" "b34" "b35" "b36" "b37" "b38" "b39" "b40" "b41"
## no weights
## vn: [1] "y"   "b1"  "b2"  "b3"  "b4"  "x1"  "b5"  "x2"  "b6"  "x3"  "b7"  "x4" 
## [13] "b8"  "x5"  "b9"  "x6"  "b10" "b11" "b12" "b13" "b14" "b15" "b16" "b17"
## [25] "b18" "b19" "b20" "b21" "b22" "b23" "b24" "b25" "b26" "b27" "b28" "b29"
## [37] "b30" "b31" "b32" "b33" "b34" "b35" "b36" "b37" "b38" "b39" "b40" "b41"
## no weights
## vn: [1] "y"   "b1"  "b2"  "b3"  "b4"  "x1"  "b5"  "x2"  "b6"  "x3"  "b7"  "x4" 
## [13] "b8"  "x5"  "b9"  "x6"  "b10" "b11" "b12" "b13" "b14" "b15" "b16" "b17"
## [25] "b18" "b19" "b20" "b21" "b22" "b23" "b24" "b25" "b26" "b27" "b28" "b29"
## [37] "b30" "b31" "b32" "b33" "b34" "b35" "b36" "b37" "b38" "b39" "b40" "b41"
## no weights
## vn: [1] "y"   "b1"  "b2"  "b3"  "b4"  "x1"  "b5"  "x2"  "b6"  "x3"  "b7"  "x4" 
## [13] "b8"  "x5"  "b9"  "x6"  "b10" "b11" "b12" "b13" "b14" "b15" "b16" "b17"
## [25] "b18" "b19" "b20" "b21" "b22" "b23" "b24" "b25" "b26" "b27" "b28" "b29"
## [37] "b30" "b31" "b32" "b33" "b34" "b35" "b36" "b37" "b38" "b39" "b40" "b41"
## no weights
## vn: [1] "y"   "b1"  "b2"  "b3"  "b4"  "x1"  "b5"  "x2"  "b6"  "x3"  "b7"  "x4" 
## [13] "b8"  "x5"  "b9"  "x6"  "b10" "b11" "b12" "b13" "b14" "b15" "b16" "b17"
## [25] "b18" "b19" "b20" "b21" "b22" "b23" "b24" "b25" "b26" "b27" "b28" "b29"
## [37] "b30" "b31" "b32" "b33" "b34" "b35" "b36" "b37" "b38" "b39" "b40" "b41"
## no weights

## initial  value 17273.889382 
## iter  20 value 7814.839497
## iter  40 value 6956.452168
## iter  60 value 3562.078201
## iter  80 value 2876.956130
## iter 100 value 2771.727490
## iter 120 value 2725.413319
## iter 140 value 2309.946563
## iter 160 value 2227.702766
## iter 180 value 2153.534988
## iter 200 value 2136.926477
## iter 220 value 2112.408209
## iter 240 value 2105.505830
## iter 260 value 2104.297916
## iter 280 value 2103.699287
## iter 300 value 2102.684227
## iter 320 value 2102.024712
## final  value 2101.647096 
## converged
## initial  value 19189.354929 
## iter  20 value 7168.639955
## iter  40 value 6306.559542
## iter  60 value 5126.481393
## iter  80 value 4885.143722
## iter 100 value 4606.138523
## iter 120 value 4421.788302
## iter 140 value 2835.098370
## iter 160 value 2532.582809
## iter 180 value 2434.206455
## iter 200 value 2356.566537
## iter 220 value 2202.087874
## iter 240 value 2173.312389
## iter 260 value 2152.990275
## iter 280 value 2150.333116
## iter 300 value 2142.629340
## iter 320 value 2132.767767
## iter 340 value 2128.817690
## iter 360 value 2128.562815
## iter 380 value 2124.399055
## iter 400 value 2121.846377
## final  value 2121.846377 
## stopped after 400 iterations
## initial  value 22682.675810 
## iter  20 value 7107.439389
## iter  40 value 6407.452943
## iter  60 value 5016.488014
## iter  80 value 4624.433805
## iter 100 value 4406.760205
## iter 120 value 4304.703921
## iter 140 value 3821.299198
## iter 160 value 3729.268331
## iter 180 value 3698.706733
## iter 200 value 3695.749947
## iter 220 value 3685.929206
## iter 240 value 3681.289933
## iter 260 value 3680.287723
## iter 280 value 3679.738781
## iter 300 value 3679.317716
## iter 320 value 3679.020653
## final  value 3678.954283 
## converged
## initial  value 17431.623442 
## iter  20 value 7982.028365
## iter  40 value 6654.263756
## iter  60 value 5120.523807
## iter  80 value 4273.500686
## iter 100 value 4182.557045
## iter 120 value 4138.598850
## iter 140 value 4080.165545
## iter 160 value 3946.228255
## iter 180 value 3618.169004
## iter 200 value 3374.211115
## iter 220 value 3334.049353
## iter 240 value 3331.314196
## iter 260 value 3312.526580
## iter 280 value 3305.626456
## iter 300 value 3303.900993
## iter 320 value 3303.518441
## iter 340 value 3302.689444
## iter 360 value 3301.605187
## iter 380 value 3301.444835
## iter 380 value 3301.444832
## final  value 3301.444547 
## converged
## initial  value 31586.837042 
## iter  20 value 7998.405040
## iter  40 value 6370.263463
## iter  60 value 4571.258707
## iter  80 value 4194.798333
## iter 100 value 3979.754828
## iter 120 value 3921.905987
## iter 140 value 3793.076128
## iter 160 value 3772.226238
## iter 180 value 3765.867233
## iter 200 value 3763.466999
## iter 220 value 3757.746328
## iter 240 value 3753.872161
## iter 260 value 3752.850981
## iter 280 value 3752.790531
## iter 300 value 3752.178415
## iter 320 value 3751.648513
## iter 340 value 3751.405254
## final  value 3751.402951 
## converged
## initial  value 23231.667172 
## iter  20 value 8248.993250
## iter  40 value 6573.172905
## iter  60 value 5255.382651
## iter  80 value 4495.747877
## iter 100 value 4343.388469
## iter 120 value 4165.253399
## iter 140 value 3464.479214
## iter 160 value 2274.256697
## iter 180 value 2136.165376
## iter 200 value 2066.458897
## iter 220 value 1907.752045
## iter 240 value 1892.091609
## iter 260 value 1885.460371
## iter 280 value 1885.186693
## iter 300 value 1884.450528
## iter 320 value 1881.350289
## iter 340 value 1880.507537
## final  value 1880.482945 
## converged
## initial  value 19436.122157 
## iter  20 value 9188.718559
## iter  40 value 8336.494118
## iter  60 value 5390.863748
## iter  80 value 5026.150880
## iter 100 value 4992.630757
## iter 120 value 4981.259370
## iter 140 value 4937.314714
## iter 160 value 4871.393409
## iter 180 value 4676.488264
## iter 200 value 4659.870554
## iter 220 value 4584.433558
## iter 240 value 4574.271196
## final  value 4574.017564 
## converged
## initial  value 27725.438510 
## iter  20 value 8528.333610
## iter  40 value 7689.246161
## iter  60 value 3970.671324
## iter  80 value 2994.272014
## iter 100 value 2763.081398
## iter 120 value 2705.952817
## iter 140 value 2637.525815
## iter 160 value 2601.802989
## iter 180 value 2497.644434
## iter 200 value 2460.944997
## iter 220 value 2159.258009
## iter 240 value 2144.580184
## iter 260 value 2130.865033
## iter 280 value 2128.186012
## iter 300 value 2120.435747
## iter 320 value 2117.243931
## iter 340 value 2116.888886
## iter 360 value 2115.888045
## iter 380 value 2115.246899
## iter 400 value 2114.980559
## final  value 2114.980559 
## stopped after 400 iterations
## initial  value 24018.488067 
## iter  20 value 7941.706394
## iter  40 value 6817.451435
## iter  60 value 3914.857157
## iter  80 value 3211.179530
## iter 100 value 2721.907140
## iter 120 value 2674.391432
## iter 140 value 2288.033016
## iter 160 value 2185.058534
## iter 180 value 2166.867293
## iter 200 value 2164.441489
## iter 220 value 2147.752990
## iter 240 value 2137.179958
## iter 260 value 2133.977720
## iter 280 value 2133.839593
## iter 280 value 2133.839581
## final  value 2133.839387 
## converged
## initial  value 22619.138407 
## iter  20 value 7846.642262
## iter  40 value 7041.317923
## iter  60 value 4875.174263
## iter  80 value 4515.778943
## iter 100 value 4360.826562
## iter 120 value 4349.378277
## iter 140 value 4264.855278
## iter 160 value 4248.154110
## final  value 4246.959304 
## converged

colnames(resall) <- pkgfunmat[,"pkg"]
rownames(resall) <- 
  paste0(rep(c("RMSE", "MSE", "MAE", "WAE", "time"), length=5*nrep), ".", rep(1:nrep, each=5))

resmin <- sapply(1:NCOL(resall),
       function(j)
       {
         idx.min <- which.min(resall[grep("RMSE.", rownames(resall)),j])
          idx <- (idx.min-1)*5+1:5
          resall[idx, j]
       })
colnames(resmin) <- pkgfunmat[,"pkg"]
rownames(resmin) <- c("RMSE", "MSE", "MAE", "WAE", "time")
kable(resmin) 

	nlsr	rminer	nnet	validann	MachineShop
RMSE	3.5482	3.5519	3.5512	3.3599	3.5467
MSE	12.5896	12.6158	12.6109	11.2892	12.5794
MAE	2.8515	2.8490	2.8562	2.6929	2.8554
WAE	13.9362	13.6683	13.8621	16.1344	14.1641
time	85.0930	11.4620	3.6490	139.0810	2.7100

which.median <- function(x)
{
  n <- length(x)
  if(n %% 2 == 1)
    which(rank(x) == n/2+1)
  else
    which(rank(x) == n/2)
}

# resmedian <- sapply(1:NCOL(resall),
#        function(j)
#        {
#          idx.min <- which.median(resall[grep("RMSE.", rownames(resall)),j])
#           idx <- (idx.min-1)*5+1:5
#           resall[idx, j]
#        })
# colnames(resmedian) <- pkgfunmat[,"pkg"]
# rownames(resmedian) <- c("RMSE", "MSE", "MAE", "WAE", "time")
# kable(resmedian)